from database.db_handler import MysqlHander
from common.my_http import MyHttp
import urllib.request
from bs4 import BeautifulSoup
from common.macro_data import MacroData
import glob
import jieba
import re
import threading
import hashlib

'''
宏观数据:商务部
'''

class ShangwubuDataTongji:
    def __init__(self):
        self.macro = MacroData()

    def v2(self, url):
        bs4_data = MyHttp.bs4_utf8_data(url)
        for ul in bs4_data.find_all(name="ul", attrs={'class':'txtList_01'}):
            for valid_data in ul.find_all(name="li"):
                try:
                    url_data = "http://www.mofcom.gov.cn" + valid_data.find("a").attrs["href"]
                    title_data = valid_data.find("a").text
                    date_data = valid_data.find("span").text.replace("[","").replace("]","")[0:10]
                    self.macro.add_macro_data(date_data, 901, title_data, url_data, "商务部")
                except Exception as e:
                    pass

    
    def get_data(self):
        self.v2("http://www.mofcom.gov.cn/article/tongjiziliao/sjtj/index.shtml")
        self.v2("http://www.mofcom.gov.cn/article/tongjiziliao/fuwzn/index.shtml")
        self.v2("http://www.mofcom.gov.cn/article/tongjiziliao/dgzz/")


def shangwubu_tongji_api():
    d = ShangwubuDataTongji()
    d.get_data()

if __name__ == '__main__':
    shangwubu_tongji_api()
